{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "36bf7ad8-320d-40c8-bd1f-19b706de060c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-21T14:01:29.195199Z",
     "iopub.status.busy": "2021-12-21T14:01:29.194601Z",
     "iopub.status.idle": "2021-12-21T14:01:36.389051Z",
     "shell.execute_reply": "2021-12-21T14:01:36.388126Z",
     "shell.execute_reply.started": "2021-12-21T14:01:29.195167Z"
    }
   },
   "outputs": [],
   "source": [
    "!pip install --upgrade parl -q\n",
    "!pip install --upgrade gym==0.10.5 -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d5e4eda7-c1c2-4f38-84a1-287786b1fec1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-21T14:01:38.412932Z",
     "iopub.status.busy": "2021-12-21T14:01:38.412135Z",
     "iopub.status.idle": "2021-12-21T14:01:54.512913Z",
     "shell.execute_reply": "2021-12-21T14:01:54.512119Z",
     "shell.execute_reply.started": "2021-12-21T14:01:38.412898Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fatal: 目标路径 'multiagent' 已经存在，并且不是一个空目录。\n"
     ]
    }
   ],
   "source": [
    "!git clone https://gitee.com/aipatchy/multiagent.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "499133e0-a085-4634-bcb2-e684014b176a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-21T14:03:55.599954Z",
     "iopub.status.busy": "2021-12-21T14:03:55.599132Z",
     "iopub.status.idle": "2021-12-21T14:03:57.700591Z",
     "shell.execute_reply": "2021-12-21T14:03:57.699937Z",
     "shell.execute_reply.started": "2021-12-21T14:03:55.599917Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m[12-21 22:03:56 MainThread @utils.py:73]\u001b[0m paddlepaddle version: 2.1.2.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/parl/remote/communication.py:38: DeprecationWarning: 'pyarrow.default_serialization_context' is deprecated as of 2.0.0 and will be removed in a future version. Use pickle or the pyarrow IPC functionality instead.\n",
      "  context = pyarrow.default_serialization_context()\n",
      "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/pyarrow/pandas_compat.py:1027: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  'floating': np.float,\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import time\n",
    "import argparse\n",
    "import numpy as np\n",
    "from parl.algorithms import MADDPG\n",
    "from parl.env.multiagent_simple_env import MAenv\n",
    "from parl.utils import logger, summary, ReplayMemory, machine_info, get_gpu_count\n",
    "\n",
    "import parl\n",
    "import paddle\n",
    "import paddle.nn as nn\n",
    "import paddle.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "34e2bbd7-1086-4f9a-9722-68f5510c705a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-26T14:11:20.534884Z",
     "iopub.status.busy": "2021-11-26T14:11:20.534546Z",
     "iopub.status.idle": "2021-11-26T14:11:20.730752Z",
     "shell.execute_reply": "2021-11-26T14:11:20.730098Z",
     "shell.execute_reply.started": "2021-11-26T14:11:20.534853Z"
    }
   },
   "outputs": [],
   "source": [
    "### 定义MODEL 与 DDPG 基本相同\n",
    "\n",
    "class MAModel(parl.Model):\n",
    "    def __init__(self, obs_dim, act_dim, critic_in_dim):\n",
    "        super(MAModel, self).__init__()\n",
    "        self.actor_model = ActorModel(obs_dim, act_dim)\n",
    "        self.critic_model = CriticModel(critic_in_dim)\n",
    "\n",
    "    def policy(self, obs):\n",
    "        return self.actor_model(obs)\n",
    "\n",
    "    def value(self, obs, act):\n",
    "        return self.critic_model(obs, act)\n",
    "\n",
    "    def get_actor_params(self):\n",
    "        return self.actor_model.parameters()\n",
    "\n",
    "    def get_critic_params(self):\n",
    "        return self.critic_model.parameters()\n",
    "\n",
    "\n",
    "class ActorModel(parl.Model):\n",
    "    def __init__(self, obs_dim, act_dim):\n",
    "        super(ActorModel, self).__init__()\n",
    "        hid1_size = 64\n",
    "        hid2_size = 64\n",
    "        self.fc1 = nn.Linear(\n",
    "            obs_dim,\n",
    "            hid1_size,\n",
    "            weight_attr=paddle.ParamAttr(\n",
    "                initializer=paddle.nn.initializer.XavierUniform()))\n",
    "        self.fc2 = nn.Linear(\n",
    "            hid1_size,\n",
    "            hid2_size,\n",
    "            weight_attr=paddle.ParamAttr(\n",
    "                initializer=paddle.nn.initializer.XavierUniform()))\n",
    "        self.fc3 = nn.Linear(\n",
    "            hid2_size,\n",
    "            act_dim,\n",
    "            weight_attr=paddle.ParamAttr(\n",
    "                initializer=paddle.nn.initializer.XavierUniform()))\n",
    "\n",
    "    def forward(self, obs):\n",
    "        hid1 = F.relu(self.fc1(obs))\n",
    "        hid2 = F.relu(self.fc2(hid1))\n",
    "        means = self.fc3(hid2)\n",
    "        return means\n",
    "\n",
    "### 不同在于 Q 函数的输入从单个的动作 a 变为所有 agent 的动作\n",
    "class CriticModel(parl.Model):\n",
    "    def __init__(self, critic_in_dim):\n",
    "        super(CriticModel, self).__init__()\n",
    "        hid1_size = 64\n",
    "        hid2_size = 64\n",
    "        out_dim = 1\n",
    "        self.fc1 = nn.Linear(\n",
    "            critic_in_dim,\n",
    "            hid1_size,\n",
    "            weight_attr=paddle.ParamAttr(\n",
    "                initializer=paddle.nn.initializer.XavierUniform()))\n",
    "        self.fc2 = nn.Linear(\n",
    "            hid1_size,\n",
    "            hid2_size,\n",
    "            weight_attr=paddle.ParamAttr(\n",
    "                initializer=paddle.nn.initializer.XavierUniform()))\n",
    "        self.fc3 = nn.Linear(\n",
    "            hid2_size,\n",
    "            out_dim,\n",
    "            weight_attr=paddle.ParamAttr(\n",
    "                initializer=paddle.nn.initializer.XavierUniform()))\n",
    "\n",
    "    def forward(self, obs_n, act_n):\n",
    "        inputs = paddle.concat(obs_n + act_n, axis=1)\n",
    "        hid1 = F.relu(self.fc1(inputs))\n",
    "        hid2 = F.relu(self.fc2(hid1))\n",
    "        Q = self.fc3(hid2)\n",
    "        Q = paddle.squeeze(Q, axis=1)\n",
    "        return Q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "bdb8f0e3-be67-4c06-82fb-f5bb842c963a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-26T14:26:38.134343Z",
     "iopub.status.busy": "2021-11-26T14:26:38.134012Z",
     "iopub.status.idle": "2021-11-26T14:26:38.149755Z",
     "shell.execute_reply": "2021-11-26T14:26:38.149136Z",
     "shell.execute_reply.started": "2021-11-26T14:26:38.134309Z"
    }
   },
   "outputs": [],
   "source": [
    "### 定义 AGENT\n",
    "### MADDPG 与 DDPG 不同, 由于需要将不同 agent 的 experience 分开, 因此 agent 自带 ReplayBuffer\n",
    "class MAAgent(parl.Agent):\n",
    "    def __init__(self,\n",
    "                 algorithm,\n",
    "                 agent_index=None,\n",
    "                 obs_dim_n=None,\n",
    "                 act_dim_n=None,\n",
    "                 batch_size=None,\n",
    "                 speedup=False):\n",
    "        assert isinstance(agent_index, int)\n",
    "        assert isinstance(obs_dim_n, list)\n",
    "        assert isinstance(act_dim_n, list)\n",
    "        assert isinstance(batch_size, int)\n",
    "        assert isinstance(speedup, bool)\n",
    "        self.agent_index = agent_index\n",
    "        self.obs_dim_n = obs_dim_n\n",
    "        self.act_dim_n = act_dim_n\n",
    "        self.batch_size = batch_size\n",
    "        self.speedup = speedup\n",
    "        self.n = len(act_dim_n)\n",
    "\n",
    "        self.memory_size = int(1e5)\n",
    "        self.min_memory_size = batch_size * 25  # batch_size * args.max_episode_len\n",
    "        self.rpm = ReplayMemory(\n",
    "            max_size=self.memory_size,\n",
    "            obs_dim=self.obs_dim_n[agent_index],\n",
    "            act_dim=self.act_dim_n[agent_index])\n",
    "        self.global_train_step = 0\n",
    "\n",
    "        super(MAAgent, self).__init__(algorithm)\n",
    "\n",
    "        # Attention: In the beginning, sync target model totally.\n",
    "        self.alg.sync_target(decay=0)\n",
    "\n",
    "    ### 输出预测动作时,每个 agent 单独行动\n",
    "    def predict(self, obs, use_target_model=False):\n",
    "        \"\"\" predict action by model or target_model\n",
    "        \"\"\"\n",
    "        obs = paddle.to_tensor(obs.reshape(1, -1), dtype='float32')\n",
    "        act = self.alg.predict(obs, use_target_model=use_target_model)\n",
    "        act_numpy = act.detach().cpu().numpy().flatten()\n",
    "        return act_numpy\n",
    "\n",
    "    ### Learn 需要获得其他 agent 的信息, 需要循环\n",
    "    def learn(self, agents):\n",
    "        \"\"\" sample batch, compute q_target and train\n",
    "        \"\"\"\n",
    "        self.global_train_step += 1\n",
    "\n",
    "        # only update parameter every 100 steps\n",
    "        if self.global_train_step % 100 != 0:\n",
    "            return 0.0\n",
    "\n",
    "        if self.rpm.size() <= self.min_memory_size:\n",
    "            return 0.0\n",
    "\n",
    "        batch_obs_n = []\n",
    "        batch_act_n = []\n",
    "        batch_obs_next_n = []\n",
    "\n",
    "        # sample batch\n",
    "        rpm_sample_index = self.rpm.make_index(self.batch_size)\n",
    "        for i in range(self.n):\n",
    "            batch_obs, batch_act, _, batch_obs_next, _ = agents[i].rpm.sample_batch_by_index(rpm_sample_index)\n",
    "            batch_obs_n.append(batch_obs)\n",
    "            batch_act_n.append(batch_act)\n",
    "            batch_obs_next_n.append(batch_obs_next)\n",
    "        _, _, batch_rew, _, batch_isOver = self.rpm.sample_batch_by_index(rpm_sample_index)\n",
    "        batch_obs_n = [paddle.to_tensor(obs, dtype='float32') for obs in batch_obs_n]\n",
    "        batch_act_n = [paddle.to_tensor(act, dtype='float32') for act in batch_act_n]\n",
    "        batch_rew = paddle.to_tensor(batch_rew, dtype='float32')\n",
    "        batch_isOver = paddle.to_tensor(batch_isOver, dtype='float32')\n",
    "\n",
    "        # compute target q\n",
    "        target_act_next_n = []\n",
    "        batch_obs_next_n = [\n",
    "            paddle.to_tensor(obs, dtype='float32') for obs in batch_obs_next_n\n",
    "        ]\n",
    "        for i in range(self.n):\n",
    "            target_act_next = agents[i].alg.predict(\n",
    "                batch_obs_next_n[i], use_target_model=True)\n",
    "            target_act_next = target_act_next.detach()\n",
    "            target_act_next_n.append(target_act_next)\n",
    "        target_q_next = self.alg.Q(\n",
    "            batch_obs_next_n, target_act_next_n, use_target_model=True)\n",
    "        target_q = batch_rew + self.alg.gamma * (\n",
    "            1.0 - batch_isOver) * target_q_next.detach()\n",
    "\n",
    "        # learn\n",
    "        critic_cost = self.alg.learn(batch_obs_n, batch_act_n, target_q)\n",
    "        critic_cost = critic_cost.cpu().detach().numpy()[0]\n",
    "\n",
    "        return critic_cost\n",
    "\n",
    "    def add_experience(self, obs, act, reward, next_obs, terminal):\n",
    "        self.rpm.append(obs, act, reward, next_obs, terminal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "fe840612-2439-4920-8357-1c42cfbc7a8c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-26T14:21:12.147301Z",
     "iopub.status.busy": "2021-11-26T14:21:12.146303Z",
     "iopub.status.idle": "2021-11-26T14:21:12.151194Z",
     "shell.execute_reply": "2021-11-26T14:21:12.150517Z",
     "shell.execute_reply.started": "2021-11-26T14:21:12.147260Z"
    }
   },
   "outputs": [],
   "source": [
    "### 参数\n",
    "CRITIC_LR = 0.01  # learning rate for the critic model\n",
    "ACTOR_LR = 0.01  # learning rate of the actor model\n",
    "GAMMA = 0.95  # reward discount factor\n",
    "TAU = 0.01  # soft update\n",
    "BATCH_SIZE = 1024\n",
    "MAX_EPISODES = 25000  # stop condition: number of episodes\n",
    "MAX_STEP_PER_EPISODE = 25  # maximum step per episode\n",
    "STAT_RATE = 1000  # statistical interval of save model or count reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "460b843f-7266-47ba-90e2-01af612ec9aa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-26T14:22:04.150455Z",
     "iopub.status.busy": "2021-11-26T14:22:04.149518Z",
     "iopub.status.idle": "2021-11-26T14:22:04.158299Z",
     "shell.execute_reply": "2021-11-26T14:22:04.157627Z",
     "shell.execute_reply.started": "2021-11-26T14:22:04.150413Z"
    }
   },
   "outputs": [],
   "source": [
    "def run_episode(env, agents):\n",
    "    obs_n = env.reset()\n",
    "    total_reward = 0\n",
    "    agents_reward = [0 for _ in range(env.n)]\n",
    "    steps = 0\n",
    "    while True:\n",
    "        steps += 1\n",
    "        ### 一次输出所有 agent 的动作\n",
    "        action_n = [agent.predict(obs) for agent, obs in zip(agents, obs_n)]\n",
    "        ### 返回所有 agent 的 (next_s , r , done)\n",
    "        next_obs_n, reward_n, done_n, _ = env.step(action_n)\n",
    "        done = all(done_n)\n",
    "        terminal = (steps >= MAX_STEP_PER_EPISODE)\n",
    "\n",
    "        # store experience\n",
    "        for i, agent in enumerate(agents):\n",
    "            agent.add_experience(obs_n[i], action_n[i], reward_n[i],\n",
    "                                 next_obs_n[i], done_n[i])\n",
    "\n",
    "        # compute reward of every agent\n",
    "        obs_n = next_obs_n\n",
    "        for i, reward in enumerate(reward_n):\n",
    "            total_reward += reward\n",
    "            agents_reward[i] += reward\n",
    "\n",
    "        # check the end of an episode\n",
    "        if done or terminal:\n",
    "            break\n",
    "\n",
    "        # show animation\n",
    "        if args.show:\n",
    "            time.sleep(0.1)\n",
    "            env.render()\n",
    "\n",
    "        # show model effect without training\n",
    "        if args.restore and args.show:\n",
    "            continue\n",
    "\n",
    "        # learn policy\n",
    "        for i, agent in enumerate(agents):\n",
    "            critic_loss = agent.learn(agents)\n",
    "            if critic_loss != 0.0:\n",
    "                summary.add_scalar('critic_loss_%d' % i, critic_loss,\n",
    "                                   agent.global_train_step)\n",
    "\n",
    "    return total_reward, agents_reward, steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "33fe4673-1d24-460c-85d2-9d899517d276",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-26T14:22:41.722634Z",
     "iopub.status.busy": "2021-11-26T14:22:41.722034Z",
     "iopub.status.idle": "2021-11-26T14:22:41.742427Z",
     "shell.execute_reply": "2021-11-26T14:22:41.741853Z",
     "shell.execute_reply.started": "2021-11-26T14:22:41.722593Z"
    }
   },
   "outputs": [],
   "source": [
    "def train_agent():\n",
    "    env = MAenv(args.env)\n",
    "    logger.info('agent num: {}'.format(env.n))\n",
    "    logger.info('observation_space: {}'.format(env.observation_space))\n",
    "    logger.info('action_space: {}'.format(env.action_space))\n",
    "    logger.info('obs_shape_n: {}'.format(env.obs_shape_n))\n",
    "    logger.info('act_shape_n: {}'.format(env.act_shape_n))\n",
    "\n",
    "    for i in range(env.n):\n",
    "        logger.info('agent {} obs_low:{} obs_high:{}'.format(\n",
    "            i, env.observation_space[i].low, env.observation_space[i].high))\n",
    "        logger.info('agent {} act_n:{}'.format(i, env.act_shape_n[i]))\n",
    "        if ('low' in dir(env.action_space[i])):\n",
    "            logger.info('agent {} act_low:{} act_high:{} act_shape:{}'.format(\n",
    "                i, env.action_space[i].low, env.action_space[i].high,\n",
    "                env.action_space[i].shape))\n",
    "            logger.info('num_discrete_space:{}'.format(\n",
    "                env.action_space[i].num_discrete_space))\n",
    "\n",
    "    from gym import spaces\n",
    "    from multiagent.multi_discrete import MultiDiscrete\n",
    "    for space in env.action_space:\n",
    "        assert (isinstance(space, spaces.Discrete)\n",
    "                or isinstance(space, MultiDiscrete))\n",
    "\n",
    "    critic_in_dim = sum(env.obs_shape_n) + sum(env.act_shape_n)\n",
    "    logger.info('critic_in_dim: {}'.format(critic_in_dim))\n",
    "\n",
    "    agents = []\n",
    "    for i in range(env.n):\n",
    "        model = MAModel(env.obs_shape_n[i], env.act_shape_n[i], critic_in_dim)\n",
    "        algorithm = MADDPG(\n",
    "            model,\n",
    "            agent_index=i,\n",
    "            act_space=env.action_space,\n",
    "            gamma=GAMMA,\n",
    "            tau=TAU,\n",
    "            critic_lr=CRITIC_LR,\n",
    "            actor_lr=ACTOR_LR)\n",
    "        agent = MAAgent(\n",
    "            algorithm,\n",
    "            agent_index=i,\n",
    "            obs_dim_n=env.obs_shape_n,\n",
    "            act_dim_n=env.act_shape_n,\n",
    "            batch_size=BATCH_SIZE,\n",
    "            speedup=(not args.restore))\n",
    "        agents.append(agent)\n",
    "    total_steps = 0\n",
    "    total_episodes = 0\n",
    "\n",
    "    episode_rewards = []  # sum of rewards for all agents\n",
    "    agent_rewards = [[] for _ in range(env.n)]  # individual agent reward\n",
    "\n",
    "    if args.restore:\n",
    "        # restore modle\n",
    "        for i in range(len(agents)):\n",
    "            model_file = args.model_dir + '/agent_' + str(i)\n",
    "            if not os.path.exists(model_file):\n",
    "                raise Exception(\n",
    "                    'model file {} does not exits'.format(model_file))\n",
    "            agents[i].restore(model_file)\n",
    "\n",
    "    t_start = time.time()\n",
    "    logger.info('Starting...')\n",
    "    while total_episodes <= MAX_EPISODES:\n",
    "        # run an episode\n",
    "        ep_reward, ep_agent_rewards, steps = run_episode(env, agents)\n",
    "        summary.add_scalar('train_reward/episode', ep_reward, total_episodes)\n",
    "        summary.add_scalar('train_reward/step', ep_reward, total_steps)\n",
    "        if args.show:\n",
    "            print('episode {}, reward {}, agents rewards {}, steps {}'.format(\n",
    "                total_episodes, ep_reward, ep_agent_rewards, steps))\n",
    "\n",
    "        # Record reward\n",
    "        total_steps += steps\n",
    "        total_episodes += 1\n",
    "        episode_rewards.append(ep_reward)\n",
    "        for i in range(env.n):\n",
    "            agent_rewards[i].append(ep_agent_rewards[i])\n",
    "\n",
    "        # Keep track of final episode reward\n",
    "        if total_episodes % STAT_RATE == 0:\n",
    "            mean_episode_reward = round(\n",
    "                np.mean(episode_rewards[-STAT_RATE:]), 3)\n",
    "            final_ep_ag_rewards = []  # agent rewards for training curve\n",
    "            for rew in agent_rewards:\n",
    "                final_ep_ag_rewards.append(round(np.mean(rew[-STAT_RATE:]), 2))\n",
    "            use_time = round(time.time() - t_start, 3)\n",
    "            logger.info(\n",
    "                'Steps: {}, Episodes: {}, Mean episode reward: {}, mean agents rewards {}, Time: {}'\n",
    "                .format(total_steps, total_episodes, mean_episode_reward,\n",
    "                        final_ep_ag_rewards, use_time))\n",
    "            t_start = time.time()\n",
    "            summary.add_scalar('mean_episode_reward/episode',\n",
    "                               mean_episode_reward, total_episodes)\n",
    "            summary.add_scalar('mean_episode_reward/step', mean_episode_reward,\n",
    "                               total_steps)\n",
    "            summary.add_scalar('use_time/1000episode', use_time,\n",
    "                               total_episodes)\n",
    "\n",
    "            # save model\n",
    "            if not args.restore:\n",
    "                model_dir = args.model_dir\n",
    "                os.makedirs(os.path.dirname(model_dir), exist_ok=True)\n",
    "                for i in range(len(agents)):\n",
    "                    model_name = '/agent_' + str(i)\n",
    "                    agents[i].save(model_dir + model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb2e9172-b626-44e5-bcb9-e5971cfea193",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-26T14:26:52.943672Z",
     "iopub.status.busy": "2021-11-26T14:26:52.942733Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m[11-26 22:26:52 MainThread @2573545636.py:3]\u001b[0m agent num: 2\n",
      "\u001b[32m[11-26 22:26:52 MainThread @2573545636.py:4]\u001b[0m observation_space: [Box(3,), Box(11,)]\n",
      "\u001b[32m[11-26 22:26:52 MainThread @2573545636.py:5]\u001b[0m action_space: [Discrete(3), Discrete(5)]\n",
      "\u001b[32m[11-26 22:26:52 MainThread @2573545636.py:6]\u001b[0m obs_shape_n: [3, 11]\n",
      "\u001b[32m[11-26 22:26:52 MainThread @2573545636.py:7]\u001b[0m act_shape_n: [3, 5]\n",
      "\u001b[32m[11-26 22:26:52 MainThread @2573545636.py:11]\u001b[0m agent 0 obs_low:[-inf -inf -inf] obs_high:[inf inf inf]\n",
      "\u001b[32m[11-26 22:26:52 MainThread @2573545636.py:12]\u001b[0m agent 0 act_n:3\n",
      "\u001b[32m[11-26 22:26:52 MainThread @2573545636.py:11]\u001b[0m agent 1 obs_low:[-inf -inf -inf -inf -inf -inf -inf -inf -inf -inf -inf] obs_high:[inf inf inf inf inf inf inf inf inf inf inf]\n",
      "\u001b[32m[11-26 22:26:52 MainThread @2573545636.py:12]\u001b[0m agent 1 act_n:5\n",
      "\u001b[32m[11-26 22:26:52 MainThread @2573545636.py:27]\u001b[0m critic_in_dim: 22\n",
      "\u001b[32m[11-26 22:26:53 MainThread @machine_info.py:88]\u001b[0m nvidia-smi -L found gpu count: 1\n",
      "\u001b[32m[11-26 22:26:53 MainThread @machine_info.py:88]\u001b[0m nvidia-smi -L found gpu count: 1\n",
      "\u001b[32m[11-26 22:26:53 MainThread @2573545636.py:64]\u001b[0m Starting...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/tensor/creation.py:125: DeprecationWarning: `np.object` is a deprecated alias for the builtin `object`. To silence this warning, use `object` by itself. Doing this will not modify any behavior and is safe. \n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  if data.dtype == np.object:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m[11-26 22:27:56 MainThread @2573545636.py:92]\u001b[0m Steps: 25000, Episodes: 1000, Mean episode reward: -150.366, mean agents rewards [-75.18, -75.18], Time: 62.978\n"
     ]
    }
   ],
   "source": [
    "if __name__ == '__main__':\n",
    "    parser = argparse.ArgumentParser()\n",
    "    # Environment\n",
    "    parser.add_argument(\n",
    "        '--env',\n",
    "        type=str,\n",
    "        default='simple_speaker_listener',\n",
    "        help='scenario of MultiAgentEnv')\n",
    "    # auto save model, optional restore model\n",
    "    parser.add_argument(\n",
    "        '--show', action='store_true', default=False, help='display or not')\n",
    "    parser.add_argument(\n",
    "        '--restore',\n",
    "        action='store_true',\n",
    "        default=False,\n",
    "        help='restore or not, must have model_dir')\n",
    "    parser.add_argument(\n",
    "        '--model_dir',\n",
    "        type=str,\n",
    "        default='./model',\n",
    "        help='directory for saving model')\n",
    "\n",
    "    # args = parser.parse_args()\n",
    "    args, unparsed = parser.parse_known_args()\n",
    "    logger.set_dir('./train_log/' + str(args.env))\n",
    "\n",
    "    train_agent()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "py35-paddle1.2.0"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
